package com.fr.crf;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;

import com.hankcs.hanlp.model.crf.CRFLexicalAnalyzer;
import com.hankcs.hanlp.seg.common.Term;

/**
 * @author 付荣
 * @version 创建时间：2018年4月8日 上午10:03:10
 */
public class CRFIKCompare1 {
	private static TransportClient transportClient;
	static {
		try {
			Settings settings = Settings.settingsBuilder().put("cluster.name", "lmm_ultimate_search_fangzhen").put("client.transport.sniff", true).build();
			transportClient = TransportClient.builder().settings(settings).build();
			transportClient.addTransportAddresses(new InetSocketTransportAddress(InetAddress.getByName("127.0.0.1"), 9300));
			transportClient.addTransportAddresses(new InetSocketTransportAddress(InetAddress.getByName("127.0.0.1"), 9302));
			transportClient.addTransportAddresses(new InetSocketTransportAddress(InetAddress.getByName("127.0.0.1"), 9306));
		} catch (UnknownHostException e) {
		}
	}

	public static void main(String[] args) throws IOException {
		CRFLexicalAnalyzer analyzer = new CRFLexicalAnalyzer();
		File file = new File("E:\\compareFZ.txt");
		FileWriter fw = new FileWriter(file);
		Map<String, Integer> crfMap = new HashMap<>();
		Map<String, Integer> ikMap = new HashMap<>();
		File f = new File("E:\\nameFZ.txt");
		FileReader fr = new FileReader(f);
		BufferedReader br = new BufferedReader(fr);
		String name = br.readLine();
		while (name != null) {
			if (StringUtils.isNotBlank(name)) {
				if(name.contains("XXXXX"))
				{
					name = br.readLine();
					continue;
				}
				name = name.replaceAll("\\pP|\\pS", "").replaceAll("\\s", "");
				AnalyzeRequest request = (new AnalyzeRequest().text(name)).analyzer("ik_max_word");
				List<AnalyzeResponse.AnalyzeToken> analyzeTokens = transportClient.admin().indices().analyze(request).actionGet().getTokens();
				List<String> ik = new ArrayList<>();
				List<String> newWordsCRF = new ArrayList<>();
				List<String> newWordsIK = new ArrayList<>();
				List<Term> seg = analyzer.seg(name);
				if (compare(analyzeTokens, seg, ik, newWordsCRF, newWordsIK)) {
					fw.append(name + "\nik:" + ik + "\ncrf:" + seg + "\nnewWordsCRF:" + newWordsCRF + "\nnewWordsIK:" + newWordsIK + "\n");
					cal(newWordsIK, ikMap);
					cal(newWordsCRF, crfMap);
				}
			}
			name = br.readLine();
		}
		List<Map.Entry<String, Integer>> crf = new ArrayList<>();
		crf.addAll(crfMap.entrySet());
		Collections.sort(crf, new Comparator<Map.Entry<String, Integer>>() {
			@Override
			public int compare(Entry<String, Integer> o1, Entry<String, Integer> o2) {
				return o2.getValue() - o1.getValue();
			}

		});
		fw.append("crf新词：\n");
		for (Map.Entry<String, Integer> entry : crf) {
			fw.append(entry.getKey() + "\t" + entry.getValue() + "\n");
		}
		List<Map.Entry<String, Integer>> ik = new ArrayList<>();
		ik.addAll(ikMap.entrySet());
		Collections.sort(ik, new Comparator<Map.Entry<String, Integer>>() {
			@Override
			public int compare(Entry<String, Integer> o1, Entry<String, Integer> o2) {
				return o2.getValue() - o1.getValue();
			}

		});
		fw.append("ik词库：\n");
		for (Map.Entry<String, Integer> entry : ik) {
			fw.append(entry.getKey() + "\t" + entry.getValue() + "\n");
		}
		fw.close();
		br.close();
		fr.close();
	}

	private static boolean compare(List<AnalyzeToken> analyzeTokens, List<Term> seg, List<String> ik, List<String> newWordsCRF, List<String> newWordsIK) {
		boolean hasNewWord = false;
		for (int i = 0; i < analyzeTokens.size(); i++) {
			ik.add(analyzeTokens.get(i).getTerm());
		}
		a: for (int i = 0; i < seg.size(); i++) {
			for (int j = 0; j < analyzeTokens.size(); j++) {
				if (seg.get(i).word.length() <= 1 || seg.get(i).word.equals(analyzeTokens.get(j).getTerm())) {
					continue a;
				}
			}
			hasNewWord = true;
			newWordsCRF.add(seg.get(i).word + "|" + seg.get(i).nature);
		}
		b: for (int i = 0; i < analyzeTokens.size(); i++) {
			for (int j = 0; j < seg.size(); j++) {
				if (analyzeTokens.get(i).getTerm().length() <= 1 || analyzeTokens.get(i).getTerm().equals(seg.get(j).word)) {
					continue b;
				}
			}
			hasNewWord = true;
			newWordsIK.add(analyzeTokens.get(i).getTerm());
		}
		return hasNewWord;
	}

	private static void cal(List<String> list, Map<String, Integer> map) {
		for (String s : list) {
			Integer i = map.get(s);
			if (i == null) {
				i = 0;
			}
			i++;
			map.put(s, i);
		}
	}
}
